# 提出问题--->文档加载--->文档分割--->embeddings词向量--->存储至向量数据库--->向量数据库根据问题检索数据--->生成含Prompt的问题--->将问题传递给LLM--->LLM给出答案

# 文档加载器
import os
import openai
import sys
import json
from langchain.chat_models import ChatOpenAI
from langchain.document_loaders import PyPDFLoader
from langchain.document_loaders.generic import GenericLoader
from langchain.document_loaders.parsers import OpenAIWhisperParser
from langchain.document_loaders.blob_loaders.youtube_audio import YoutubeAudioLoader
from langchain.document_loaders import WebBaseLoader
from langchain.document_loaders import NotionDirectoryLoader
import sys
sys.path.append('E:/ffmpeg/ffmpeg-6.0-essentials_build/bin')


api_key = "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"
os.environ['OPENAI_API_KEY'] = "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"


def load_pdf():
    # 1、加载PDF文档
    # 创建一个 PyPDFLoader Class 实例，输入为待加载的pdf文档路径
    loader = PyPDFLoader("docs/cs229_lectures/MachineLearning-Lecture01.pdf")
    # 调用 PyPDFLoader Class 的函数 load对pdf文件进行加载
    pages = loader.load()

    # 2、探索加载数据
    print(type(pages))  # 文档集合
    print(type(pages[0]))  # 单个文档
    print(len(pages))  # 页数
    print(pages[0].page_content[0:500])  # 文档内容
    print(pages[0].metadata)  # 描述性数据


def load_video():
    url = "https://www.youtube.com/watch?v=jGwO_UgTS7I"
    save_dir = "docs/youtube/"
    # 创建一个 GenericLoader Class 实例
    loader = GenericLoader(
        # 将链接url中的Youtube视频的音频下载下来,存在本地路径save_dir
        YoutubeAudioLoader([url], save_dir),
        # 使用OpenAIWhisperPaser解析器将音频转化为文本
        OpenAIWhisperParser()
    )
    # 调用 GenericLoader Class 的函数 load对视频的音频文件进行加载
    docs = loader.load()


def load_url():
    # 创建一个 WebBaseLoader Class 实例
    url = "https://github.com/basecamp/handbook/blob/master/37signals-is-you.md"
    header = {'User-Agent': 'python-requests/2.27.1',
              'Accept-Encoding': 'gzip, deflate, br',
              'Accept': '*/*',
              'Connection': 'keep-alive'}
    loader = WebBaseLoader(web_path=url, header_template=header)
    # 调用 WebBaseLoader Class 的函数 load对文件进行加载
    docs = loader.load()
    print(type(docs))
    print(len(docs))
    doc = docs[0]
    print(type(doc))
    print(doc.page_content)
    # convert_to_json = json.loads(doc.page_content)
    # extracted_markdow = convert_to_json['payload']['blob']['richText']
    # print(extracted_markdow)


def load_notion():
    loader = NotionDirectoryLoader("docs/Notion_DB")
    docs = loader.load()
    print(type(docs))
    print(len(docs))
    doc = docs[0]
    print(type(doc))
    print(doc.page_content[0:500])


if __name__ == '__main__':
    # load_pdf()
    load_video()
    # load_url()
    # load_notion()
